{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "coated-teddy",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from scipy import stats\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "plt.rcParams['font.family']='SimHei'\n",
    "plt.rcParams['axes.unicode_minus']= False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "placed-mumbai",
   "metadata": {},
   "source": [
    "## 读取csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "combined-drill",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>timestamp</th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>converted</th>\n",
       "      <th>date</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>851104</td>\n",
       "      <td>2017-01-21 22:11:48.556739</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-01-21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>804228</td>\n",
       "      <td>2017-01-12 08:01:45.159739</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-01-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>661590</td>\n",
       "      <td>2017-01-11 16:55:06.154213</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-01-11</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>853541</td>\n",
       "      <td>2017-01-08 18:28:03.143765</td>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>0</td>\n",
       "      <td>2017-01-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>864975</td>\n",
       "      <td>2017-01-21 01:52:26.210827</td>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>1</td>\n",
       "      <td>2017-01-21</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id                   timestamp      group landing_page  converted  \\\n",
       "0   851104  2017-01-21 22:11:48.556739    control     old_page          0   \n",
       "1   804228  2017-01-12 08:01:45.159739    control     old_page          0   \n",
       "2   661590  2017-01-11 16:55:06.154213  treatment     new_page          0   \n",
       "3   853541  2017-01-08 18:28:03.143765  treatment     new_page          0   \n",
       "4   864975  2017-01-21 01:52:26.210827    control     old_page          1   \n",
       "\n",
       "         date  \n",
       "0  2017-01-21  \n",
       "1  2017-01-12  \n",
       "2  2017-01-11  \n",
       "3  2017-01-08  \n",
       "4  2017-01-21  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data = pd.read_csv('./ab_data.csv',sep=',')\n",
    "data['date'] = data.timestamp.str[:10]\n",
    "data.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "royal-dublin",
   "metadata": {},
   "source": [
    "## 计算统计量"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "conventional-documentary",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_1 = data.groupby(['group','landing_page','date'],as_index=False)['user_id'].count()\n",
    "data_2 = data.groupby(['group','landing_page','date'],as_index=False)['converted'].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "cooked-atlanta",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>date</th>\n",
       "      <th>user_id</th>\n",
       "      <th>converted</th>\n",
       "      <th>ctr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-02</td>\n",
       "      <td>35</td>\n",
       "      <td>3</td>\n",
       "      <td>0.085714</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-03</td>\n",
       "      <td>94</td>\n",
       "      <td>10</td>\n",
       "      <td>0.106383</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-04</td>\n",
       "      <td>77</td>\n",
       "      <td>8</td>\n",
       "      <td>0.103896</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-05</td>\n",
       "      <td>111</td>\n",
       "      <td>9</td>\n",
       "      <td>0.081081</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-06</td>\n",
       "      <td>78</td>\n",
       "      <td>16</td>\n",
       "      <td>0.205128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>87</th>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-20</td>\n",
       "      <td>88</td>\n",
       "      <td>11</td>\n",
       "      <td>0.125000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>88</th>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-21</td>\n",
       "      <td>92</td>\n",
       "      <td>13</td>\n",
       "      <td>0.141304</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>89</th>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-22</td>\n",
       "      <td>79</td>\n",
       "      <td>8</td>\n",
       "      <td>0.101266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90</th>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-23</td>\n",
       "      <td>95</td>\n",
       "      <td>11</td>\n",
       "      <td>0.115789</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>91</th>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-24</td>\n",
       "      <td>46</td>\n",
       "      <td>1</td>\n",
       "      <td>0.021739</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>92 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        group landing_page        date  user_id  converted       ctr\n",
       "0     control     new_page  2017-01-02       35          3  0.085714\n",
       "1     control     new_page  2017-01-03       94         10  0.106383\n",
       "2     control     new_page  2017-01-04       77          8  0.103896\n",
       "3     control     new_page  2017-01-05      111          9  0.081081\n",
       "4     control     new_page  2017-01-06       78         16  0.205128\n",
       "..        ...          ...         ...      ...        ...       ...\n",
       "87  treatment     old_page  2017-01-20       88         11  0.125000\n",
       "88  treatment     old_page  2017-01-21       92         13  0.141304\n",
       "89  treatment     old_page  2017-01-22       79          8  0.101266\n",
       "90  treatment     old_page  2017-01-23       95         11  0.115789\n",
       "91  treatment     old_page  2017-01-24       46          1  0.021739\n",
       "\n",
       "[92 rows x 6 columns]"
      ]
     },
     "execution_count": 86,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ctr = pd.concat([data_1,data_2['converted']],axis=1)\n",
    "ctr['ctr'] = ctr['converted']/ctr['user_id']\n",
    "ctr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "id": "mounted-arena",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>landing_page</th>\n",
       "      <th>date</th>\n",
       "      <th>user_id</th>\n",
       "      <th>converted</th>\n",
       "      <th>ctr</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>control</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td>95</td>\n",
       "      <td>14</td>\n",
       "      <td>0.147368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>36</th>\n",
       "      <td>control</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td>6714</td>\n",
       "      <td>809</td>\n",
       "      <td>0.120494</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>treatment</td>\n",
       "      <td>new_page</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td>6549</td>\n",
       "      <td>743</td>\n",
       "      <td>0.113452</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>82</th>\n",
       "      <td>treatment</td>\n",
       "      <td>old_page</td>\n",
       "      <td>2017-01-15</td>\n",
       "      <td>91</td>\n",
       "      <td>14</td>\n",
       "      <td>0.153846</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        group landing_page        date  user_id  converted       ctr\n",
       "13    control     new_page  2017-01-15       95         14  0.147368\n",
       "36    control     old_page  2017-01-15     6714        809  0.120494\n",
       "59  treatment     new_page  2017-01-15     6549        743  0.113452\n",
       "82  treatment     old_page  2017-01-15       91         14  0.153846"
      ]
     },
     "execution_count": 88,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 抽取某一天的ctr，01-15\n",
    "n_o = ctr[ctr.date == \"2017-01-15\"]\n",
    "n_o"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "informed-eclipse",
   "metadata": {},
   "source": [
    "## 计算统计量的显著性P值"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "id": "immediate-candle",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.047121647191827966"
      ]
     },
     "execution_count": 89,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sigma = np.sqrt(ctr.ctr[2]*(1-ctr.ctr[2])/ctr.user_id[2] + ctr.ctr[1]*(1-ctr.ctr[1])/ctr.user_id[1])\n",
    "sigma"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "governing-tamil",
   "metadata": {},
   "outputs": [],
   "source": [
    "t = ctr.ctr[2]-ctr.ctr[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "valued-optics",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5210446623381935"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "p = 1-stats.norm.cdf(t,0,sigma)\n",
    "p"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "instant-trustee",
   "metadata": {},
   "source": [
    "## 用统计量的显著性P值与显著性α比较做决策"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "green-patrol",
   "metadata": {},
   "outputs": [],
   "source": [
    "alpha = 0.05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "violent-affect",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "显著性P>α，实验组点击率 <= 对照组\n"
     ]
    }
   ],
   "source": [
    "if p > alpha:\n",
    "    print(\"显著性P>α，实验组点击率 <= 对照组\")\n",
    "else:\n",
    "    print(\"显著性P<α，实验组点击率 > 对照组\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "aquatic-observer",
   "metadata": {},
   "source": [
    "## 封装"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "listed-opposition",
   "metadata": {},
   "outputs": [],
   "source": [
    "def abtest(df: pd.DataFrame, alpha=0.05, group_col: str = None, value_col: str =\n",
    "None):\n",
    "   \n",
    "    if not group_col:\n",
    "        group_col = df.columns[0]\n",
    "    if not value_col:\n",
    "        value_col = df.columns[1]\n",
    "\n",
    "    temp = df.groupby(group_col,as_index=False)[value_col].mean()\n",
    "    temp_n = df.groupby(group_col,as_index=False)[value_col].count()\n",
    "    tongjiliang = temp.iloc[0,1]-temp.iloc[1,1]\n",
    "\n",
    "    diff_error = np.sqrt(temp.iloc[0,1]*(1-temp.iloc[0,1])/temp_n.iloc[0,1]+temp_n.iloc[1,1]*(1-temp_n.iloc[1,1])/temp_n.iloc[1,1])\n",
    "\n",
    "    tongjiliang_left_p = stats.norm.cdf(tongjiliang,0,diff_error)\n",
    "    tongjiliang_right_p = 1 - stats.norm.cdf(tongjiliang,0,diff_error)\n",
    "    tongjiliang_site_p = tongjiliang_left_p*2\n",
    "\n",
    "    if tongjiliang_site_p > 1:\n",
    "        tongjiliang_site_p = tongjiliang_right_p*2\n",
    "        \n",
    "    temp_1 = [[temp.iloc[0,0],temp.iloc[1,0],tongjiliang,'左侧',tongjiliang_left_p,np.where(tongjiliang_left_p<alpha,'显著','不显著')],\n",
    "            [temp.iloc[0,0],temp.iloc[1,0],tongjiliang,'右侧',tongjiliang_right_p,np.where(tongjiliang_right_p<alpha,'显著','不显著')],\n",
    "            [temp.iloc[0,0],temp.iloc[1,0],tongjiliang,'双侧',tongjiliang_site_p,np.where(tongjiliang_left_p<alpha,'显著','不显著')]]\n",
    "\n",
    "\n",
    "    temp = pd.DataFrame(temp_1,columns =['p','p0','统计量','检测','P_valuer','结果'])\n",
    "    return temp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "consistent-makeup",
   "metadata": {},
   "outputs": [],
   "source": [
    "abtest(df_C)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "spare-attack",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>group</th>\n",
       "      <th>converted</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>treatment</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>73</th>\n",
       "      <td>treatment</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>132</th>\n",
       "      <td>control</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>133</th>\n",
       "      <td>control</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>134</th>\n",
       "      <td>control</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         group  converted\n",
       "9    treatment          1\n",
       "73   treatment          0\n",
       "132    control          0\n",
       "133    control          0\n",
       "134    control          0"
      ]
     },
     "execution_count": 103,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "temp = data[data.date == '2017-01-15'].loc[((data.group == 'control')&(data.landing_page == 'old_page'))\n",
    "    | ((data.group == 'treatment')&(data.landing_page == 'new_page')),['group','converted']]\n",
    "temp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "id": "commercial-engagement",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>p</th>\n",
       "      <th>p0</th>\n",
       "      <th>统计量</th>\n",
       "      <th>检测</th>\n",
       "      <th>P_valuer</th>\n",
       "      <th>结果</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>control</td>\n",
       "      <td>treatment</td>\n",
       "      <td>0.007042</td>\n",
       "      <td>左侧</td>\n",
       "      <td>NaN</td>\n",
       "      <td>不显著</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>control</td>\n",
       "      <td>treatment</td>\n",
       "      <td>0.007042</td>\n",
       "      <td>右侧</td>\n",
       "      <td>NaN</td>\n",
       "      <td>不显著</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>control</td>\n",
       "      <td>treatment</td>\n",
       "      <td>0.007042</td>\n",
       "      <td>双侧</td>\n",
       "      <td>NaN</td>\n",
       "      <td>不显著</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         p         p0       统计量  检测  P_valuer   结果\n",
       "0  control  treatment  0.007042  左侧       NaN  不显著\n",
       "1  control  treatment  0.007042  右侧       NaN  不显著\n",
       "2  control  treatment  0.007042  双侧       NaN  不显著"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "abtest(temp)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
